package util;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/*这个类作用是处理采集下的网页内容，使用了正则表达式截取出需要采集的网址*/

public class SpiderRobot {

	public void run() {
		// 程序出错后重启的时间间隔
		final long errWating = 20000;
		// 搜索引擎的编号
		Connection conn = null;
		Statement stmt = null;
		Connection connCheck = null;
		Statement stmtCheck = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		// workingID是个标记位，指示在现在处理到哪一位了
		int workingID = 0;
		String keyword = null;
		String search = "sec" + sourceNet;
		
		while (true) {
			try {
				Class.forName("com.mysql.jdbc.Driver");
				conn = DriverManager.getConnection("jdbc:mysql://" + host + "/" + dBName + "?user=root");
				connGetSite = conn;
				connCheck = conn;
				stmtGetSite = connGetSite.createStatement();
				stmtCheck = connCheck.createStatement();
				// 查询出需要处理的网址，check是标记为0就是需要处理的
				rsGetSite = stmtGetSite.executeQuery("select * from keywordnofilter where " + search + " = 0");
				while (rsGetSite.next()) {
					workingID = rsGetSite.getInt("id");
					String tag = rsGetSite.getString("keyword");
					keyword = URLEncoder.encode(tag, codeNo).replaceAll("\\%2B", "+");
					// 抓取网址开始的时间点
					Long btime = System.currentTimeMillis();
					SpiderBodyXML sbx = new SpiderBodyXML();
					sbx.setRepeatTimes(repeatTimes);
					sbx.SpiderPageToXML(first + keyword + end);
					String SitesString = sbx.getWebSiteBodyString();
					sbx.clear();
					sbx = null;
					// 到这里已经处理完了当下的keywords，需要修改filter字段
					stmt = conn.createStatement();
					// 修改标记放在处理抓取下来的字符串之前目的就是如果这条数据出错了，下次在重启时也不需要再处理了，虽然少处理一天数据但保证了程序的连续性
					stmtCheck.execute("update keywordnofilter set " + search + " = 1 where id = " + workingID);
					// 如果是0表示链接有错误返回的只是默认值
					if (SitesString == "0") {
						// 出错的没有成功抓取的连接放入badurl表
						stmt.execute("insert into badurl (id,badid,search) values (0," + workingID + ",'" + search + "')");

					} else {
						Pattern p = Pattern.compile(urlPattern);
						Matcher m = p.matcher(SitesString);
						int tagURLController = 0;
						while (m.find()) {
							String url = m.group(2);
							if(head.length()>2){
								url = head + url.replaceAll("&amp;", "&");
							}
							// 过长的url处理，抓取.html .shtml等结尾前的部分，如果没有就跳出这次循环这条数据不要了
							if (url.length() > 1000) {
								Pattern LongPattern = Pattern.compile("http://.*\\.shtml|http://.*\\.html|http://.*\\.htm|http://.*\\.asp|http://.*\\.aspx|http://.*\\.httm|http://.*\\.bottom|http://.*\\.xhtml|http://.*\\.shtm|http://.*\\.php|http://.*\\.jsp|http://.*\\.jhtml|http://.*\\.dhtml");
								Matcher LongMatcher = LongPattern.matcher(url);
								LongMatcher.reset();
								if (LongMatcher.find()) {
									url = LongMatcher.group();
								} else {
									continue;
								}

							}
							// 这是了重复判断机制，如果有重复的就不加入了，这就使中断后再开始采集时不会出现重复数据
							rs = stmt.executeQuery("select id from tagurl where url = '" + url + "'");
							if (!rs.next()) {
								
								/*// 再用一次正则表达式截取出仅包含域名部分的网址，正则的意义是:字母+/就是域名的结束位
								Pattern webSitePattern = Pattern.compile("[a-z]/");
								Matcher webSiteMatcher = webSitePattern.matcher(url);
								webSiteMatcher.reset();
								// 如果能够匹配出主页的网址内容插入数据库，匹配不出就不插入，匹配不出的是ip地址网页质量不会太高,continue跳过这一条
								if (webSiteMatcher.find()) {
									// 字母+/的开始位置再加2个正好就是网址的结束位置
									int j = webSiteMatcher.start() + 2;
									String netPort = new String();
									netPort = url.substring(0, j);
									// 下面一块是为了处理二级子域名博客，有些博客使用子域名，单独处理
									String[] subName = { "blog.sohu.com", "blog.hexun.com", "blogbus.com", "blog.163.com", "blog.edu.cn", "user.qzone.qq.com","iteye.com" };
									for (int i = 0; i < 7; i++) {
										Pattern subNamePattern = Pattern.compile(subName[i]);
										Matcher subNameMatcher = subNamePattern.matcher(netPort);
										if (subNameMatcher.find()) {
											netPort = "http://" + subName[i] + "/";
										}
									}
									// 处理子域名代码结束	
									
									stmt.execute("insert into tagurl (id,tag,url,urlnotail,subId,spContent,sec2,sec3,sec4,sec5,sec6,sec7,sec8,sec9,sec10,sec11,sec12,sec13,sec14,sourceNet,sequence) values (0,'" + tag + "','" + url + "','" + netPort + "'," + rsGetSite.getInt("subId") + ",0," + rsGetSite.getInt("sec2") + "," + rsGetSite.getInt("sec3") + "," + rsGetSite.getInt("sec4") + "," + rsGetSite.getInt("sec5") + "," + rsGetSite.getInt("sec6") + "," + rsGetSite.getInt("sec7") + "," + rsGetSite.getInt("sec8") + "," + rsGetSite.getInt("sec9") + "," + rsGetSite.getInt("sec10") + "," + rsGetSite.getInt("sec11") + "," + rsGetSite.getInt("sec12") + "," + rsGetSite.getInt("sec13") + "," + rsGetSite.getInt("sec14") + ",'" + search
											+ "'," + tagURLController + ")");
								} else {
									continue;
								}*/
								stmt.execute("insert into tagurl (id,tag,url,sourceNet,subId,spider,pub,deltag,sequence) values (0,'" + tag + "','" + url + "'," + sourceNet + "," + rsGetSite.getInt("subId") + ",0,0,0," + tagURLController + ")");
							}
							tagURLController++;
							System.out.println(tagURLController);
						}
						// tagURLController=0也就是一个链接也没抓取到，把这个页计入到badurl表中
						if (tagURLController == 0) {
							stmt.execute("insert into badurl (id,badid) values (0," + workingID + ")");
						}
					}
					Long etime = System.currentTimeMillis();
					// 打印出处理使用的时间
					System.out.println(dBName + "----" + search + "----" + workingID + "----使用了" + (etime - btime) + "毫秒----" + new Date());
					// 如果不到10秒就加一段延时
					if ((etime - btime) < interval) {
						Thread.sleep(interval - (etime - btime));
					}
				}

				break;
			}
			// 所有exception的处理方式都是打印错误名字后再休眠一会重新启动
			catch (ClassNotFoundException e) {
				System.out.println("-----ClassNotFoundException------");
				// e.printStackTrace();
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					System.out.println("-----InterruptedException------");
					e.printStackTrace();
				}
			} catch (UnsupportedEncodingException e) {
				System.out.println("-----UnsupportedEncodingException------");
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					System.out.println("-----InterruptedException------");
					e.printStackTrace();
				}
			} catch (SQLException e) {

				// 处理
				System.out.println("-----SQLException------");
				e.printStackTrace();
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					System.out.println("-----InterruptedException------");
					e.printStackTrace();
				}

			} catch (InterruptedException e) {

				// 处理
				System.out.println("-----InterruptedException------");
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					e.printStackTrace();
				}

			} finally {
				try {
					if (rs != null) {
						rs.close();
						rs = null;
					}
					if (stmt != null) {
						stmt.close();
						stmt = null;
					}
					if (conn != null) {
						conn.close();
						conn = null;
					}
					if (rsGetSite != null) {
						rsGetSite.close();
						rsGetSite = null;
					}
					if (stmtGetSite != null) {
						stmtGetSite.close();
						stmtGetSite = null;
					}
					if (connGetSite != null) {
						connGetSite.close();
						connGetSite = null;
					}
					if (stmtCheck != null) {
						stmtCheck.close();
						stmtCheck = null;
					}
					if (connCheck != null) {
						connCheck.close();
						connCheck = null;
					}
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	String urlPattern = null;
	String host = null;
	String dBName = null;
	String first = null;
	String codeNo = null;
	String head = null;
	int sourceNet = 0;
	

	public void setSourceNet(int sourceNet) {
		this.sourceNet = sourceNet; 
	}

	public void setHead(String head) {
		this.head = head;
	}

	public void setCodeNo(String codeNo) {
		this.codeNo = codeNo;
	}

	public void setFirst(String first) {
		this.first = first;
	}

	String end = null;

	public void setEnd(String end) {
		this.end = end;
	}

	int interval = 0;
	int repeatTimes = 0;

	public void setRepeatTimes(int repeatTimes) {
		this.repeatTimes = repeatTimes;
	}

	public void setInterval(int interval) {
		this.interval = interval;
	}

	public void setHost(String host) {
		this.host = host;
	}

	public void setdBName(String dBName) {
		this.dBName = dBName;
	}

	public void setUrlPattern(String urlPattern) {
		this.urlPattern = urlPattern;
	}
}
